{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flow chart for dirty zillow pipeline\n",
    "This notebook contains code to create a flow chart of the dirty zillow pipeline and all preprocessing necessary to run both the clean zillow and dirty zillow experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prerequisites: You need flowgiston and graphviz installed, on Mac OS X use e.g.\n",
    "# !brew install graphviz\n",
    "# !pip3 install flowgiston"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Pipeline processing definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import pandas as pd\n",
    "import re\n",
    "from flowgiston import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data/zillow_dirty.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper row object to allow fancy integer and column based string access within UDFs!\n",
    "class Row:\n",
    "    def __init__(self, data, columns=None):\n",
    "        assert(isinstance(data, (tuple, list)))\n",
    "        assert(isinstance(columns, (tuple, list)) or columns is None)\n",
    "        self.data = tuple(data)\n",
    "        self.columns = tuple(columns[:len(data)]) if columns is not None else None \n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        # check for int also works for bool!\n",
    "        if isinstance(key, int):\n",
    "            return self.data[key]\n",
    "        elif isinstance(key, str):\n",
    "            if self.columns is None:\n",
    "                raise KeyError(\"no columns defined, can't access column '{}'\".format(key))\n",
    "            elif key not in self.columns:\n",
    "                raise KeyError(\"could not find column column '{}'\".format(key))\n",
    "            return self.data[self.columns.index(key)]\n",
    "        else:\n",
    "            raise IndexError()\n",
    "            \n",
    "    def __repr__(self):\n",
    "        if self.columns:\n",
    "            if len(self.columns) < len(self.data):\n",
    "                self.columns = self.columns + [None] * (len(self.data) - len(self.columns))\n",
    "            return '(' + ','.join(['{}={}'.format(c, d) for c, d in zip(self.columns, self.data)]) + ')'\n",
    "        else:\n",
    "            return '(' + ','.join(['{}'.format(d) for d in self.data]) + ')'\n",
    "# recursive expansion of Row objects potentially present in data.\n",
    "def expand_row(x):\n",
    "    # Note: need to use here type construction, because isinstance fails for dict input when checking for list\n",
    "    if hasattr(type(x), '__iter__') and not isinstance(x, str):\n",
    "        if type(x) is tuple:\n",
    "            return tuple([expand_row(el) for el in x])\n",
    "        elif type(x) is list:\n",
    "            return [expand_row(el) for el in x]\n",
    "        elif type(x) is dict:\n",
    "            return {expand_row(key) : expand_row(val) for key, val in x.items()}\n",
    "        else:\n",
    "            raise TypeError(\"custom sequence type used, can't convert to data representation\")\n",
    "    return x.data if isinstance(x, Row) else x\n",
    "\n",
    "def result_to_row(res, columns=None):\n",
    "    # convert result to row object, i.e. deal with unpacking etc.\n",
    "    # is result a dict?\n",
    "    if type(res) is dict:\n",
    "        # are all keys strings? If so, then unpack!\n",
    "        # else, keep it as dict return object!\n",
    "        if all(map(lambda k: type(k) == str, res.keys())):\n",
    "            # columns become keys, values \n",
    "            columns = res.keys()\n",
    "            data = tuple(map(lambda k: res[k], columns))\n",
    "            return Row(data, columns)\n",
    "    \n",
    "    \n",
    "    # is it a row object?\n",
    "    # => convert to tuple!\n",
    "    r = expand_row(res)\n",
    "    \n",
    "    if type(r) is not tuple:\n",
    "        r = (r,)\n",
    "    else:\n",
    "        if len(r) == 0:\n",
    "            r ((),) # special case, empty tuple\n",
    "    \n",
    "    return Row(r, columns)\n",
    "\n",
    "def apply_func(f, row):\n",
    "    if len(row.data) != 1:\n",
    "        return f(row)\n",
    "    else:\n",
    "        try:\n",
    "            return f(row)\n",
    "        except TypeError as te:\n",
    "            msg = str(te)\n",
    "            # single op error?\n",
    "            # try unwrapped...\n",
    "            return f(row.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pure python class to create dataflow!\n",
    "\n",
    "class Dataset:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.es = []\n",
    "        self.ns = []\n",
    "        self.name = None\n",
    "        self.parent = None\n",
    "    \n",
    "    @classmethod\n",
    "    def from_csv(cls, path):\n",
    "        # special conversion function for boolean necessary\n",
    "        def to_bool(value):\n",
    "            valid = {'true':True, 't':True, 'yes':True, 'y':True, 'false':False, 'f':False, 'no':False, 'n':False,              }   \n",
    "        \n",
    "            if isinstance(value, bool):\n",
    "                return value\n",
    "        \n",
    "            if not isinstance(value, str):\n",
    "                raise ValueError('invalid literal for boolean. Not a string.')\n",
    "        \n",
    "            lower_value = value.lower()\n",
    "            if lower_value in valid:\n",
    "                return valid[lower_value]\n",
    "            else:\n",
    "                raise ValueError('invalid literal for boolean: \"%s\"' % value)\n",
    "        \n",
    "        def parse(s):\n",
    "            assert isinstance(s, str)\n",
    "            # try to parse s as different types\n",
    "            if s in ['',]:\n",
    "                return None\n",
    "            try:\n",
    "                return to_bool(s)\n",
    "            except:\n",
    "                pass\n",
    "            try:\n",
    "                return int(s.strip())\n",
    "            except:\n",
    "                pass\n",
    "            try:\n",
    "                return float(s.strip())\n",
    "            except:\n",
    "                pass\n",
    "            try:\n",
    "                return json.loads(s.strip())\n",
    "            except:\n",
    "                pass\n",
    "            # return as string, final option remaining...\n",
    "            return s\n",
    "        \n",
    "        ds = cls()\n",
    "        ds.name = \"readcsv\"\n",
    "        \n",
    "        with open(data_path, newline='') as csvfile:\n",
    "            reader = csv.DictReader(csvfile)\n",
    "            for i, csv_row in enumerate(reader):\n",
    "                parsed_row = [parse(el) for el in csv_row.values()]\n",
    "                row = Row(parsed_row + [i], list(csv_row.keys()) + ['row_number'])\n",
    "                ds.ns.append(row)\n",
    "        return ds\n",
    "    \n",
    "    def withColumn(self, column, ftor):\n",
    "        ds = Dataset()\n",
    "        ds.parent = self\n",
    "        ds.name = 'withColumn'\n",
    "        ds.column = column\n",
    "        \n",
    "        for row in self.ns.copy():\n",
    "            # go over all rows and process into normal set or exception set!\n",
    "            try:\n",
    "                call_res = apply_func(ftor, row)\n",
    "                if column in row.columns:\n",
    "                    col_idx = row.columns.index(column)\n",
    "                    tmp = list(row.data)\n",
    "                    tmp[col_idx] = expand_row(call_res)\n",
    "                    row.data = tuple(tmp)\n",
    "                else:\n",
    "                    row.columns = row.columns + (column,) if row.columns is not None else tuple([None] * len(row.data), column)\n",
    "                    row.data = row.data + result_to_row(call_res).data\n",
    "                    \n",
    "                ds.ns.append(row)\n",
    "            except Exception as e:\n",
    "                entry = {'exception': e, 'ename' : type(e).__name__, 'row': row}\n",
    "                ds.es.append(entry)\n",
    "        \n",
    "        return ds\n",
    "    \n",
    "    def filter(self, ftor):\n",
    "        ds = Dataset()\n",
    "        ds.parent = self\n",
    "        ds.name = 'filter'\n",
    "        for row in self.ns.copy():\n",
    "            try:\n",
    "                call_res = apply_func(ftor, row)\n",
    "                if call_res:\n",
    "                    ds.ns.append(row)\n",
    "            except Exception as e:\n",
    "                entry = {'exception': e, 'ename' : type(e).__name__, 'row': row}\n",
    "                ds.es.append(entry)\n",
    "        return ds\n",
    "    \n",
    "    def mapColumn(self, column, ftor):\n",
    "        ds = Dataset()\n",
    "        ds.parent = self\n",
    "        ds.name = 'mapColumn'\n",
    "        ds.column = column\n",
    "        \n",
    "        for row in self.ns.copy():\n",
    "            try:\n",
    "                col_idx = row.columns.index(column)\n",
    "                arg = row.data[col_idx]\n",
    "                call_res = f(arg)\n",
    "                tmp = list(row.data)\n",
    "                tmp[col_idx] = expand_row(call_res)\n",
    "                row.data = tuple(tmp)\n",
    "\n",
    "                ds.ns.append(row)\n",
    "            except Exception as e:\n",
    "                entry = {'exception': e, 'ename' : type(e).__name__, 'row': row}\n",
    "                ds.es.append(entry)\n",
    "        return ds\n",
    "    \n",
    "    def ignore(self, exception):\n",
    "        ds = Dataset()\n",
    "        ds.parent = self\n",
    "        ds.ns = self.ns.copy()\n",
    "        ds.name = 'ignore'\n",
    "        \n",
    "        ds.es = list(filter(lambda x: exception.__name__ != x['ename'], self.es))\n",
    "        \n",
    "        ds.ignored = len(self.es) - len(ds.es)\n",
    "        \n",
    "        return ds\n",
    "    \n",
    "    def resolve(self, exception, ftor):\n",
    "        ds = Dataset()\n",
    "        ds.parent = self\n",
    "        ds.ns = self.ns.copy()\n",
    "        ds.name = 'resolve'\n",
    "        \n",
    "        dummy_ds = Dataset()\n",
    "        dummy_ds.ns = []\n",
    "        \n",
    "        for entry in self.es:\n",
    "            if exception.__name__ == entry['ename']:\n",
    "                dummy_ds.ns.append(entry['row'])\n",
    "            else:\n",
    "                ds.es.append(entry)\n",
    "                \n",
    "        # call on dummy ds same operation using ftor!\n",
    "        p = self\n",
    "        while p is not None and p.name == 'resolve' or p.name == 'ignore':\n",
    "            p = p.parent\n",
    "        normal_name = p.name\n",
    "        res_ds = None\n",
    "        if normal_name == 'withColumn':\n",
    "            res_ds = dummy_ds.withColumn(p.column, ftor)\n",
    "        elif normal_name == 'mapColumn':\n",
    "            res_ds = dummy_ds.mapColumn(p.column, ftor)\n",
    "        else:\n",
    "            raise Exception('op {} not supported yet!'.format(normal_name))\n",
    "            \n",
    "        ds.num_resolved = len(res_ds.ns)\n",
    "        ds.num_throwing = len(res_ds.es)\n",
    "            \n",
    "        ds.ns += res_ds.ns\n",
    "        ds.es += res_ds.es\n",
    "                \n",
    "        return ds\n",
    "    \n",
    "    def selectColumns(self, columns):\n",
    "        ds = Dataset()\n",
    "        ds.parent = self\n",
    "        ds.name = 'selectColumns'\n",
    "        \n",
    "        for row in self.ns:\n",
    "            proj_row = Row(tuple([row[name] for name in columns]), columns.copy())\n",
    "            ds.ns.append(proj_row)\n",
    "        \n",
    "        return ds\n",
    "        \n",
    "    # left to implement: resolve, ignore, filter, mapColumn, selectColumns.\n",
    "    # + charting!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import re\n",
    "\n",
    "def extractBd(x):\n",
    "    val = x['facts and features']\n",
    "    max_idx = val.find(' bd')\n",
    "    if max_idx < 0:\n",
    "        max_idx = len(val)\n",
    "    s = val[:max_idx]\n",
    "\n",
    "    # find comma before\n",
    "    split_idx = s.rfind(',')\n",
    "    if split_idx < 0:\n",
    "        split_idx = 0\n",
    "    else:\n",
    "        split_idx += 2\n",
    "    r = s[split_idx:]\n",
    "    return int(r)\n",
    "\n",
    "def resolveBd(x):\n",
    "    if 'Studio' in x['facts and features']:\n",
    "        return 1\n",
    "    raise ValueError\n",
    "    \n",
    "def extractType(x):\n",
    "    t = x['title'].lower()\n",
    "    type = 'unknown'\n",
    "    if 'condo' in t or 'apartment' in t:\n",
    "        type = 'condo'\n",
    "    if 'house' in t:\n",
    "        type = 'house'\n",
    "    return type\n",
    "\n",
    "def extractBa(x):\n",
    "    val = x['facts and features']\n",
    "    max_idx = val.find(' ba')\n",
    "    if max_idx < 0:\n",
    "        max_idx = len(val)\n",
    "    s = val[:max_idx]\n",
    "\n",
    "    # find comma before\n",
    "    split_idx = s.rfind(',')\n",
    "    if split_idx < 0:\n",
    "        split_idx = 0\n",
    "    else:\n",
    "        split_idx += 2\n",
    "    r = s[split_idx:]\n",
    "    ba = math.ceil(2.0 * float(r)) / 2.0\n",
    "    return ba\n",
    "\n",
    "def extractSqft(x):\n",
    "    val = x['facts and features']\n",
    "    max_idx = val.find(' sqft')\n",
    "    if max_idx < 0:\n",
    "        max_idx = len(val)\n",
    "    s = val[:max_idx]\n",
    "\n",
    "    split_idx = s.rfind('ba ,')\n",
    "    if split_idx < 0:\n",
    "        split_idx = 0\n",
    "    else:\n",
    "        split_idx += 5\n",
    "    r = s[split_idx:]\n",
    "    r = r.replace(',', '')\n",
    "    return int(r)\n",
    "\n",
    "def extractOffer(x):\n",
    "    offer = x['title'].lower()\n",
    "    if 'sale' in offer:\n",
    "        return 'sale'\n",
    "    if 'rent' in offer:\n",
    "        return 'rent'\n",
    "    if 'sold' in offer:\n",
    "        return 'sold'\n",
    "    if 'foreclose' in offer.lower():\n",
    "        return 'foreclosed'\n",
    "    return offer\n",
    "\n",
    "def extractPrice(x):\n",
    "    price = x['price']\n",
    "    p = 0\n",
    "    if x['offer'] == 'sold':\n",
    "        # price is to be calculated using price/sqft * sqft\n",
    "        val = x['facts and features']\n",
    "        s = val[val.find('Price/sqft:') + len('Price/sqft:') + 1:]\n",
    "        r = s[s.find('$')+1:s.find(', ') - 1]\n",
    "        price_per_sqft = int(r)\n",
    "        p = price_per_sqft * x['sqft']\n",
    "    elif x['offer'] == 'rent':\n",
    "        max_idx = price.rfind('/')\n",
    "        p = int(price[1:max_idx].replace(',', ''))\n",
    "    else:\n",
    "        # take price from price column\n",
    "        p = int(price[1:].replace(',', ''))\n",
    "\n",
    "    return p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zillow_pipeline(data_path, exclude_resolvers=False):\n",
    "    \"\"\" process updated zillow pipeline yielding condos for sale \"\"\"\n",
    "    ds = [Dataset.from_csv(data_path)]\n",
    "    ds.append(ds[-1].withColumn('bedrooms', extractBd))\n",
    "    if not exclude_resolvers:\n",
    "        ds.append(ds[-1].resolve(ValueError, resolveBd))\n",
    "        ds.append(ds[-1].ignore(ValueError))\n",
    "    ds.append(ds[-1].filter(lambda x: x['bedrooms'] < 10))\n",
    "    ds.append(ds[-1].withColumn(\"type\", extractType))\n",
    "    ds.append(ds[-1].filter(lambda x: x['type'] == 'condo'))\n",
    "    ds.append(ds[-1].withColumn(\"zipcode\", lambda x: '%05d' % int(x['postal_code'])))\n",
    "    if not exclude_resolvers:\n",
    "        ds.append(ds[-1].ignore(TypeError))\n",
    "    ds.append(ds[-1].withColumn(\"bathrooms\", extractBa))\n",
    "    if not exclude_resolvers:\n",
    "        ds.append(ds[-1].ignore(ValueError))\n",
    "    ds.append(ds[-1].withColumn(\"sqft\", extractSqft))\n",
    "    ds.append(ds[-1].ignore(ValueError))\n",
    "    ds.append(ds[-1].withColumn(\"offer\", extractOffer))\n",
    "    ds.append(ds[-1].withColumn(\"price\", extractPrice))\n",
    "    if not exclude_resolvers:\n",
    "        ds.append(ds[-1].resolve(ValueError, lambda x: int(re.sub('[^0-9.]*', '', x['price']))))\n",
    "    ds.append(ds[-1].filter(lambda x: 100000 < x['price'] < 2e7 and x['offer'] == 'sale'))\n",
    "    ds.append(ds[-1].selectColumns([\"url\", \"zipcode\", \"address\", \"city\", \"state\",\n",
    "                             \"bedrooms\", \"bathrooms\", \"sqft\", \"offer\", \"type\", \"price\"]))\n",
    "    return ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Create flow chart for original dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process to get indices & graph\n",
    "data_path = '../data/zillow_dirty.csv'\n",
    "steps = zillow_pipeline(data_path)\n",
    "output_indices = set(map(lambda x: x['row_number'], steps[-2].ns))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example errors happening in the dirty zillow pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(title=For Rent,address=None,city=None,state=None,postal_code=None,price=None,facts and features=1 $1,500+,real estate provider=None,url=https://www.zillow.com/b/35-Prospect-St-WOBURN-MA/42.477146,-71.148444_ll/,sales_date=Updated today,row_number=48)\n",
      "(title=For Rent,address=None,city=None,state=None,postal_code=None,price=None,facts and features=1 $1,500+,real estate provider=None,url=https://www.zillow.com/b/35-Prospect-St-WOBURN-MA/42.477146,-71.148444_ll/,sales_date=Updated today,row_number=48)\n",
      "(title=One Canal Apartment Homes,address=None,city=None,state=None,postal_code=None,price=None,facts and features=0 $2,605 2 $7,295 3 $7,009,real estate provider=None,url=https://www.zillow.com/b/one-canal-apartment-homes-boston-ma-5Xj76y/,sales_date=Updated yesterday,row_number=3797,bedrooms=9,type=condo)\n",
      "(title=Apartment For Rent,address=19 Chestnut St APT 3B,city=Boston,state=MA,postal_code=2108.0,price=$2,750/mo,facts and features=1 bd , -- ba , 625 sqft,real estate provider=None,url=https://www.zillow.com/homedetails/19-Chestnut-St-APT-3B-Boston-MA-02108/2089956168_zpid/,sales_date=13 days ago,row_number=1867,bedrooms=1,type=condo,zipcode=02108)\n",
      "(title=Apartment For Rent,address=Woburn,city=Woburn,state=MA,postal_code=1801.0,price=$2,000/mo,facts and features=2 bds , 1 ba , -- sqft,real estate provider=None,url=https://www.zillow.com/homedetails/Woburn-Woburn-MA-01801/2096189436_zpid/,sales_date=1 day ago,row_number=56,bedrooms=2,type=condo,zipcode=01801,bathrooms=1.0)\n",
      "(title=Apartment For Rent,address=105 Chelsea St # 4HK7UN4QT,city=Boston,state=MA,postal_code=2128.0,price=2900,facts and features=3 bds , 1 ba , 925 sqft,real estate provider=None,url=https://www.zillow.com/homedetails/105-Chelsea-St-4HK7UN4QT-Boston-MA-02128/2089835249_zpid/,sales_date=7 days ago,row_number=12991,bedrooms=3,type=condo,zipcode=02128,bathrooms=1.0,sqft=925,offer=rent)\n"
     ]
    }
   ],
   "source": [
    "for step in steps:\n",
    "    if len(step.es) > 0:\n",
    "        print(step.es[0]['row'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Overall flow of exceptions/normal rows printed as flow chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.44.1 (20200629.0846)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"298pt\" height=\"1610pt\"\n",
       " viewBox=\"0.00 0.00 297.50 1610.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1606)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-1606 293.5,-1606 293.5,4 -4,4\"/>\n",
       "<!-- n_35d18b6f85c14451ade89a9391c52091 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>n_35d18b6f85c14451ade89a9391c52091</title>\n",
       "<ellipse fill=\"lightblue\" stroke=\"black\" cx=\"48.5\" cy=\"-18\" rx=\"29.5\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">tocsv</text>\n",
       "</g>\n",
       "<!-- n_1dfb6335b35e4432b34f74eb59ceecbe -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>n_1dfb6335b35e4432b34f74eb59ceecbe</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"97,-123 0,-123 0,-87 97,-87 97,-123\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\">selectColumns</text>\n",
       "</g>\n",
       "<!-- n_1dfb6335b35e4432b34f74eb59ceecbe&#45;&gt;n_35d18b6f85c14451ade89a9391c52091 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>n_1dfb6335b35e4432b34f74eb59ceecbe&#45;&gt;n_35d18b6f85c14451ade89a9391c52091</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-86.8C48.5,-75.16 48.5,-59.55 48.5,-46.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-46.18 48.5,-36.18 45,-46.18 52,-46.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1231 normal</text>\n",
       "</g>\n",
       "<!-- n_d9b7076793d94159983190868f8d28c7 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>n_d9b7076793d94159983190868f8d28c7</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-210 21.5,-210 21.5,-174 75.5,-174 75.5,-210\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_d9b7076793d94159983190868f8d28c7&#45;&gt;n_1dfb6335b35e4432b34f74eb59ceecbe -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>n_d9b7076793d94159983190868f8d28c7&#45;&gt;n_1dfb6335b35e4432b34f74eb59ceecbe</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-173.8C48.5,-162.16 48.5,-146.55 48.5,-133.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-133.18 48.5,-123.18 45,-133.18 52,-133.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1231 normal</text>\n",
       "</g>\n",
       "<!-- n_bd20d6946edd46ada9d47daf3caad514 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>n_bd20d6946edd46ada9d47daf3caad514</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"180.5,-297 124.5,-297 124.5,-261 180.5,-261 180.5,-297\"/>\n",
       "<text text-anchor=\"middle\" x=\"152.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\">resolve</text>\n",
       "</g>\n",
       "<!-- n_bd20d6946edd46ada9d47daf3caad514&#45;&gt;n_d9b7076793d94159983190868f8d28c7 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>n_bd20d6946edd46ada9d47daf3caad514&#45;&gt;n_d9b7076793d94159983190868f8d28c7</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M131.45,-260.8C115.95,-248.13 94.67,-230.73 77.54,-216.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.47,-213.79 69.51,-210.18 75.04,-219.21 79.47,-213.79\"/>\n",
       "<text text-anchor=\"middle\" x=\"174\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;8 resolved, 0 exception</text>\n",
       "</g>\n",
       "<!-- n_e54587ea5e264d76abae479a941cd3c9 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>n_e54587ea5e264d76abae479a941cd3c9</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-384 6,-384 6,-348 91,-348 91,-384\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_e54587ea5e264d76abae479a941cd3c9&#45;&gt;n_d9b7076793d94159983190868f8d28c7 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>n_e54587ea5e264d76abae479a941cd3c9&#45;&gt;n_d9b7076793d94159983190868f8d28c7</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-347.99C44.57,-334.26 42.39,-314.42 41.5,-297 40.68,-281.02 40.68,-276.98 41.5,-261 42.19,-247.52 43.65,-232.61 45.06,-220.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-220.35 46.26,-210.01 41.61,-219.53 48.56,-220.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 6355 normal</text>\n",
       "</g>\n",
       "<!-- n_e54587ea5e264d76abae479a941cd3c9&#45;&gt;n_bd20d6946edd46ada9d47daf3caad514 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>n_e54587ea5e264d76abae479a941cd3c9&#45;&gt;n_bd20d6946edd46ada9d47daf3caad514</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.55,-347.8C85.05,-335.13 106.33,-317.73 123.46,-303.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.96,-306.21 131.49,-297.18 121.53,-300.79 125.96,-306.21\"/>\n",
       "<text text-anchor=\"middle\" x=\"140\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 8 exception</text>\n",
       "</g>\n",
       "<!-- n_30f9e5d8801544c696ca1c44a2e8552c -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>n_30f9e5d8801544c696ca1c44a2e8552c</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-471 6,-471 6,-435 91,-435 91,-471\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_30f9e5d8801544c696ca1c44a2e8552c&#45;&gt;n_e54587ea5e264d76abae479a941cd3c9 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>n_30f9e5d8801544c696ca1c44a2e8552c&#45;&gt;n_e54587ea5e264d76abae479a941cd3c9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-434.8C48.5,-423.16 48.5,-407.55 48.5,-394.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-394.18 48.5,-384.18 45,-394.18 52,-394.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 6363 normal</text>\n",
       "</g>\n",
       "<!-- n_bc1979a24ed94a4d9e77b06d2056b412 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>n_bc1979a24ed94a4d9e77b06d2056b412</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"178.5,-558 124.5,-558 124.5,-522 178.5,-522 178.5,-558\"/>\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_bc1979a24ed94a4d9e77b06d2056b412&#45;&gt;n_30f9e5d8801544c696ca1c44a2e8552c -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>n_bc1979a24ed94a4d9e77b06d2056b412&#45;&gt;n_30f9e5d8801544c696ca1c44a2e8552c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M130.66,-521.8C115.3,-509.13 94.22,-491.73 77.26,-477.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.25,-474.84 69.31,-471.18 74.8,-480.24 79.25,-474.84\"/>\n",
       "<text text-anchor=\"middle\" x=\"145.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;5047 ignored</text>\n",
       "</g>\n",
       "<!-- n_3d9854d8f9a74a5881c69a80b0f2583b -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>n_3d9854d8f9a74a5881c69a80b0f2583b</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-645 6,-645 6,-609 91,-609 91,-645\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_3d9854d8f9a74a5881c69a80b0f2583b&#45;&gt;n_30f9e5d8801544c696ca1c44a2e8552c -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>n_3d9854d8f9a74a5881c69a80b0f2583b&#45;&gt;n_30f9e5d8801544c696ca1c44a2e8552c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-608.99C44.57,-595.26 42.39,-575.42 41.5,-558 40.68,-542.02 40.68,-537.98 41.5,-522 42.19,-508.52 43.65,-493.61 45.06,-481.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-481.35 46.26,-471.01 41.61,-480.53 48.56,-481.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 6363 normal</text>\n",
       "</g>\n",
       "<!-- n_3d9854d8f9a74a5881c69a80b0f2583b&#45;&gt;n_bc1979a24ed94a4d9e77b06d2056b412 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>n_3d9854d8f9a74a5881c69a80b0f2583b&#45;&gt;n_bc1979a24ed94a4d9e77b06d2056b412</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.34,-608.8C84.7,-596.13 105.78,-578.73 122.74,-564.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.2,-567.24 130.69,-558.18 120.75,-561.84 125.2,-567.24\"/>\n",
       "<text text-anchor=\"middle\" x=\"149.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 5047 exception</text>\n",
       "</g>\n",
       "<!-- n_a014efb531294eb780beabc22f2063a1 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>n_a014efb531294eb780beabc22f2063a1</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"185.5,-732 131.5,-732 131.5,-696 185.5,-696 185.5,-732\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_a014efb531294eb780beabc22f2063a1&#45;&gt;n_3d9854d8f9a74a5881c69a80b0f2583b -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>n_a014efb531294eb780beabc22f2063a1&#45;&gt;n_3d9854d8f9a74a5881c69a80b0f2583b</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M136.24,-695.8C119.69,-683.01 96.92,-665.42 78.73,-651.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"80.78,-648.52 70.73,-645.18 76.5,-654.06 80.78,-648.52\"/>\n",
       "<text text-anchor=\"middle\" x=\"146\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;419 ignored</text>\n",
       "</g>\n",
       "<!-- n_fa08dd6ad0d74b8193e76a7474aec055 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>n_fa08dd6ad0d74b8193e76a7474aec055</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-819 6,-819 6,-783 91,-783 91,-819\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_fa08dd6ad0d74b8193e76a7474aec055&#45;&gt;n_3d9854d8f9a74a5881c69a80b0f2583b -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>n_fa08dd6ad0d74b8193e76a7474aec055&#45;&gt;n_3d9854d8f9a74a5881c69a80b0f2583b</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-782.99C44.57,-769.26 42.39,-749.42 41.5,-732 40.68,-716.02 40.68,-711.98 41.5,-696 42.19,-682.52 43.65,-667.61 45.06,-655.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-655.35 46.26,-645.01 41.61,-654.53 48.56,-655.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"82\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 11410 normal</text>\n",
       "</g>\n",
       "<!-- n_fa08dd6ad0d74b8193e76a7474aec055&#45;&gt;n_a014efb531294eb780beabc22f2063a1 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>n_fa08dd6ad0d74b8193e76a7474aec055&#45;&gt;n_a014efb531294eb780beabc22f2063a1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M70.76,-782.8C87.31,-770.01 110.08,-752.42 128.27,-738.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"130.5,-741.06 136.27,-732.18 126.22,-735.52 130.5,-741.06\"/>\n",
       "<text text-anchor=\"middle\" x=\"150\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 419 exception</text>\n",
       "</g>\n",
       "<!-- n_fe91f63d4efc472db19b5aadaa572b5d -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>n_fe91f63d4efc472db19b5aadaa572b5d</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"185.5,-906 131.5,-906 131.5,-870 185.5,-870 185.5,-906\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_fe91f63d4efc472db19b5aadaa572b5d&#45;&gt;n_fa08dd6ad0d74b8193e76a7474aec055 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>n_fe91f63d4efc472db19b5aadaa572b5d&#45;&gt;n_fa08dd6ad0d74b8193e76a7474aec055</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M136.24,-869.8C119.69,-857.01 96.92,-839.42 78.73,-825.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"80.78,-822.52 70.73,-819.18 76.5,-828.06 80.78,-822.52\"/>\n",
       "<text text-anchor=\"middle\" x=\"139.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;3 ignored</text>\n",
       "</g>\n",
       "<!-- n_ebbb56a887574e999dfa12e31ee95374 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>n_ebbb56a887574e999dfa12e31ee95374</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-993 6,-993 6,-957 91,-957 91,-993\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_ebbb56a887574e999dfa12e31ee95374&#45;&gt;n_fa08dd6ad0d74b8193e76a7474aec055 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>n_ebbb56a887574e999dfa12e31ee95374&#45;&gt;n_fa08dd6ad0d74b8193e76a7474aec055</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-956.99C44.57,-943.26 42.39,-923.42 41.5,-906 40.68,-890.02 40.68,-885.98 41.5,-870 42.19,-856.52 43.65,-841.61 45.06,-829.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-829.35 46.26,-819.01 41.61,-828.53 48.56,-829.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"82\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 11829 normal</text>\n",
       "</g>\n",
       "<!-- n_ebbb56a887574e999dfa12e31ee95374&#45;&gt;n_fe91f63d4efc472db19b5aadaa572b5d -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>n_ebbb56a887574e999dfa12e31ee95374&#45;&gt;n_fe91f63d4efc472db19b5aadaa572b5d</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M70.76,-956.8C87.31,-944.01 110.08,-926.42 128.27,-912.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"130.5,-915.06 136.27,-906.18 126.22,-909.52 130.5,-915.06\"/>\n",
       "<text text-anchor=\"middle\" x=\"143\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 3 exception</text>\n",
       "</g>\n",
       "<!-- n_3f601488ced94e4bb260510ed4f4c553 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>n_3f601488ced94e4bb260510ed4f4c553</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-1080 21.5,-1080 21.5,-1044 75.5,-1044 75.5,-1080\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_3f601488ced94e4bb260510ed4f4c553&#45;&gt;n_ebbb56a887574e999dfa12e31ee95374 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>n_3f601488ced94e4bb260510ed4f4c553&#45;&gt;n_ebbb56a887574e999dfa12e31ee95374</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1043.8C48.5,-1032.16 48.5,-1016.55 48.5,-1003.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1003.18 48.5,-993.18 45,-1003.18 52,-1003.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 11832 normal</text>\n",
       "</g>\n",
       "<!-- n_9d74d68b53f24b67a0c4deefa662008c -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>n_9d74d68b53f24b67a0c4deefa662008c</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-1167 6,-1167 6,-1131 91,-1131 91,-1167\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_9d74d68b53f24b67a0c4deefa662008c&#45;&gt;n_3f601488ced94e4bb260510ed4f4c553 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>n_9d74d68b53f24b67a0c4deefa662008c&#45;&gt;n_3f601488ced94e4bb260510ed4f4c553</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1130.8C48.5,-1119.16 48.5,-1103.55 48.5,-1090.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1090.18 48.5,-1080.18 45,-1090.18 52,-1090.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 34690 normal</text>\n",
       "</g>\n",
       "<!-- n_c1626ba937c44a699700cd3d31714f57 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>n_c1626ba937c44a699700cd3d31714f57</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-1254 21.5,-1254 21.5,-1218 75.5,-1218 75.5,-1254\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1232.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_c1626ba937c44a699700cd3d31714f57&#45;&gt;n_9d74d68b53f24b67a0c4deefa662008c -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>n_c1626ba937c44a699700cd3d31714f57&#45;&gt;n_9d74d68b53f24b67a0c4deefa662008c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1217.8C48.5,-1206.16 48.5,-1190.55 48.5,-1177.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1177.18 48.5,-1167.18 45,-1177.18 52,-1177.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1188.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 34690 normal</text>\n",
       "</g>\n",
       "<!-- n_3b150d99905d4ec3b9ddc053e4c06581 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>n_3b150d99905d4ec3b9ddc053e4c06581</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"138.5,-1341 84.5,-1341 84.5,-1305 138.5,-1305 138.5,-1341\"/>\n",
       "<text text-anchor=\"middle\" x=\"111.5\" y=\"-1319.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_3b150d99905d4ec3b9ddc053e4c06581&#45;&gt;n_c1626ba937c44a699700cd3d31714f57 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>n_3b150d99905d4ec3b9ddc053e4c06581&#45;&gt;n_c1626ba937c44a699700cd3d31714f57</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M98.75,-1304.8C89.7,-1292.59 77.4,-1275.99 67.24,-1262.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"70,-1260.13 61.23,-1254.18 64.37,-1264.29 70,-1260.13\"/>\n",
       "<text text-anchor=\"middle\" x=\"123.5\" y=\"-1275.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;3426 ignored</text>\n",
       "</g>\n",
       "<!-- n_860c61a913dc425da5d856210ea40df6 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>n_860c61a913dc425da5d856210ea40df6</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"140.5,-1428 84.5,-1428 84.5,-1392 140.5,-1392 140.5,-1428\"/>\n",
       "<text text-anchor=\"middle\" x=\"112.5\" y=\"-1406.3\" font-family=\"Times,serif\" font-size=\"14.00\">resolve</text>\n",
       "</g>\n",
       "<!-- n_860c61a913dc425da5d856210ea40df6&#45;&gt;n_3b150d99905d4ec3b9ddc053e4c06581 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>n_860c61a913dc425da5d856210ea40df6&#45;&gt;n_3b150d99905d4ec3b9ddc053e4c06581</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M113.89,-1391.87C114.49,-1382.22 114.97,-1369.94 114.5,-1359 114.39,-1356.44 114.24,-1353.78 114.05,-1351.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"117.54,-1350.76 113.26,-1341.07 110.56,-1351.32 117.54,-1350.76\"/>\n",
       "<text text-anchor=\"middle\" x=\"202\" y=\"-1362.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;1286 resolved, 3426 exception</text>\n",
       "</g>\n",
       "<!-- n_39a01161aa1d4347b8eefb6f82be293f -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>n_39a01161aa1d4347b8eefb6f82be293f</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-1515 6,-1515 6,-1479 91,-1479 91,-1515\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1493.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_39a01161aa1d4347b8eefb6f82be293f&#45;&gt;n_c1626ba937c44a699700cd3d31714f57 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>n_39a01161aa1d4347b8eefb6f82be293f&#45;&gt;n_c1626ba937c44a699700cd3d31714f57</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.95,-1478.72C40.3,-1455.17 32.41,-1411.61 29.5,-1374 26.54,-1335.75 34.68,-1291.87 41.21,-1264.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"44.7,-1264.78 43.7,-1254.23 37.9,-1263.1 44.7,-1264.78\"/>\n",
       "<text text-anchor=\"middle\" x=\"70\" y=\"-1362.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 33858 normal</text>\n",
       "</g>\n",
       "<!-- n_39a01161aa1d4347b8eefb6f82be293f&#45;&gt;n_860c61a913dc425da5d856210ea40df6 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>n_39a01161aa1d4347b8eefb6f82be293f&#45;&gt;n_860c61a913dc425da5d856210ea40df6</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M61.45,-1478.8C70.65,-1466.59 83.14,-1449.99 93.47,-1436.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"96.35,-1438.27 99.57,-1428.18 90.76,-1434.06 96.35,-1438.27\"/>\n",
       "<text text-anchor=\"middle\" x=\"127.5\" y=\"-1449.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 4712 exception</text>\n",
       "</g>\n",
       "<!-- n_4d65166172384a03b78142d29d12fc2f -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>n_4d65166172384a03b78142d29d12fc2f</title>\n",
       "<ellipse fill=\"lightblue\" stroke=\"black\" cx=\"48.5\" cy=\"-1584\" rx=\"37.89\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1580.3\" font-family=\"Times,serif\" font-size=\"14.00\">readcsv</text>\n",
       "</g>\n",
       "<!-- n_4d65166172384a03b78142d29d12fc2f&#45;&gt;n_39a01161aa1d4347b8eefb6f82be293f -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>n_4d65166172384a03b78142d29d12fc2f&#45;&gt;n_39a01161aa1d4347b8eefb6f82be293f</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1565.8C48.5,-1554.16 48.5,-1538.55 48.5,-1525.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1525.18 48.5,-1515.18 45,-1525.18 52,-1525.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1536.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 38570 normal</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<flowgiston.base.FlowgistonChart at 0x128e21700>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create flow graph\n",
    "node = steps[-1]\n",
    "\n",
    "def make_chart(node):\n",
    "    Base = flowgiston_base(fillcolor='lightblue')\n",
    "\n",
    "    class OpNode(Base):\n",
    "        style='filled'\n",
    "        shape='box'\n",
    "\n",
    "    class ExceptNode(Base):\n",
    "        style='filled'\n",
    "        shape='box'\n",
    "        fillcolor = 'lightyellow'\n",
    "\n",
    "    chart = FlowgistonChart(Base)\n",
    "    last_node = chart.end('tocsv')\n",
    "\n",
    "    last_except_path_node = None\n",
    "    while node is not None:\n",
    "        if node.name != 'resolve' and node.name != 'ignore':\n",
    "            new_last_node = chart.OpNode.node(node.name) if node.parent else chart.start(node.name)\n",
    "            new_last_node.edge(last_node, ' {} normal'.format(len(node.ns))) # how many inflowing from ns of parent?\n",
    "            last_node = new_last_node\n",
    "\n",
    "            if last_except_path_node is not None:\n",
    "                new_last_node.edge(last_except_path_node, ' {} exception'.format(len(node.es)))\n",
    "                last_except_path_node = None\n",
    "            node = node.parent\n",
    "        else:\n",
    "\n",
    "            # go except chain up!\n",
    "            except_node = last_node\n",
    "            while node is not None and node.name == 'resolve' or node.name == 'ignore':\n",
    "                next_except_node = chart.ExceptNode.node(node.name)\n",
    "                edge_label = ''\n",
    "\n",
    "                if node.name == 'resolve':\n",
    "                    edge_label = '  {} resolved, {} exception'.format(node.num_resolved, node.num_throwing)\n",
    "                elif node.name == 'ignore':\n",
    "                    edge_label ='  {} ignored'.format(node.ignored)\n",
    "                next_except_node.edge(except_node, edge_label)\n",
    "                except_node = next_except_node\n",
    "                last_except_path_node = except_node\n",
    "                node = node.parent\n",
    "    return chart\n",
    "\n",
    "chart = make_chart(node)\n",
    "chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'zillow_dirty_flow_graph.pdf'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# render file to pdf\n",
    "chart.render('zillow_dirty_flow_graph')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Obtain row numbers for normal rows and bad rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data/zillow_dirty.csv'\n",
    "steps = zillow_pipeline(data_path, exclude_resolvers=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "normal_indices = set(map(lambda r: r['row_number'], steps[0].ns))\n",
    "except_indices = set()\n",
    "for step in steps:\n",
    "    if len(step.ns) > 0 and 'row_number' in step.ns[0].columns:\n",
    "        normal_indices = normal_indices & set(map(lambda r: r['row_number'], step.ns))\n",
    "        except_indices |= set(map(lambda e: e['row']['row_number'], step.es))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There should be 9635 except indices:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9635"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(except_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and 1211 normal rows as final output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1211"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(normal_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to generate synthetic file, let's save indices to file in sorted order:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(sorted(except_indices), columns=['bad_indices'])\n",
    "df.to_csv('../data/exception_rows_indices.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above pipeline, we see that there are two operators using resolve functions. Which rows do they resolve?\n",
    "We use these rows to replace original bad rows in the file for the synthetic generation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps = zillow_pipeline(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "resolver resolved 1286\n",
      "withColumn 4712\n",
      "resolve 3426\n",
      "resolver resolved 8\n",
      "withColumn 8\n",
      "resolve 0\n"
     ]
    }
   ],
   "source": [
    "except_resolved_indices = set()\n",
    "for step in steps:\n",
    "    if step.name == 'resolve':\n",
    "        parent_es_indices = set(map(lambda e: e['row']['row_number'], step.parent.es))\n",
    "        es_indices = set(map(lambda e: e['row']['row_number'], step.es))\n",
    "        \n",
    "        # which es indices did resolve actually resolve?\n",
    "        resolved_indices = parent_es_indices - es_indices\n",
    "        print('resolver resolved {}'.format(len(resolved_indices)))\n",
    "        print(step.parent.name, len(step.parent.es))\n",
    "        print(step.name, len(step.es))\n",
    "        except_resolved_indices |= resolved_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1294"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(except_resolved_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To avoid the averaging out effect, let's only use rows which make it to the final output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1231"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_indices = list(map(lambda r: r['row_number'], steps[-2].ns))\n",
    "len(output_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "except_resolved_indices = list(set(output_indices) & set(except_resolved_indices))\n",
    "len(except_resolved_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write indices to file as well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(sorted(except_resolved_indices), columns=['resolved_indices'])\n",
    "df.to_csv('../data/exception_rows_resolved_indices.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Checking whether synthetic data and clean data are correctly working\n",
    "#### 4.1 synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> reading input files\n",
      ">>> substituting lines\n",
      ">>> replicating data 1x\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00, 25.04it/s]\n",
      "done.\n"
     ]
    }
   ],
   "source": [
    "!cd .. && python3 preprocess-data.py -s 1 -m synth -o data/zillow_synthetic.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data/zillow_synthetic.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps = zillow_pipeline(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.44.1 (20200629.0846)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"278pt\" height=\"1610pt\"\n",
       " viewBox=\"0.00 0.00 277.50 1610.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1606)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-1606 273.5,-1606 273.5,4 -4,4\"/>\n",
       "<!-- n_b7de47f8654d48a1a7d0fa7c67e36812 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>n_b7de47f8654d48a1a7d0fa7c67e36812</title>\n",
       "<ellipse fill=\"lightblue\" stroke=\"black\" cx=\"48.5\" cy=\"-18\" rx=\"29.5\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">tocsv</text>\n",
       "</g>\n",
       "<!-- n_385e087001b040368a18ef25df4b8d7b -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>n_385e087001b040368a18ef25df4b8d7b</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"97,-123 0,-123 0,-87 97,-87 97,-123\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\">selectColumns</text>\n",
       "</g>\n",
       "<!-- n_385e087001b040368a18ef25df4b8d7b&#45;&gt;n_b7de47f8654d48a1a7d0fa7c67e36812 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>n_385e087001b040368a18ef25df4b8d7b&#45;&gt;n_b7de47f8654d48a1a7d0fa7c67e36812</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-86.8C48.5,-75.16 48.5,-59.55 48.5,-46.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-46.18 48.5,-36.18 45,-46.18 52,-46.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 10846 normal</text>\n",
       "</g>\n",
       "<!-- n_96543e44dd914a4cbb24145a22ad374c -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>n_96543e44dd914a4cbb24145a22ad374c</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-210 21.5,-210 21.5,-174 75.5,-174 75.5,-210\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_96543e44dd914a4cbb24145a22ad374c&#45;&gt;n_385e087001b040368a18ef25df4b8d7b -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>n_96543e44dd914a4cbb24145a22ad374c&#45;&gt;n_385e087001b040368a18ef25df4b8d7b</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-173.8C48.5,-162.16 48.5,-146.55 48.5,-133.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-133.18 48.5,-123.18 45,-133.18 52,-133.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 10846 normal</text>\n",
       "</g>\n",
       "<!-- n_10938372722d4602b4cd027a64a821d3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>n_10938372722d4602b4cd027a64a821d3</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"187.5,-297 131.5,-297 131.5,-261 187.5,-261 187.5,-297\"/>\n",
       "<text text-anchor=\"middle\" x=\"159.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\">resolve</text>\n",
       "</g>\n",
       "<!-- n_10938372722d4602b4cd027a64a821d3&#45;&gt;n_96543e44dd914a4cbb24145a22ad374c -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>n_10938372722d4602b4cd027a64a821d3&#45;&gt;n_96543e44dd914a4cbb24145a22ad374c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M137.04,-260.8C120.34,-248.01 97.36,-230.42 79,-216.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"81,-213.48 70.93,-210.18 76.74,-219.03 81,-213.48\"/>\n",
       "<text text-anchor=\"middle\" x=\"178\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 resolved, 0 exception</text>\n",
       "</g>\n",
       "<!-- n_d7feef994af44c2389164a5d22907b54 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>n_d7feef994af44c2389164a5d22907b54</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-384 6,-384 6,-348 91,-348 91,-384\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_d7feef994af44c2389164a5d22907b54&#45;&gt;n_96543e44dd914a4cbb24145a22ad374c -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>n_d7feef994af44c2389164a5d22907b54&#45;&gt;n_96543e44dd914a4cbb24145a22ad374c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-347.99C44.57,-334.26 42.39,-314.42 41.5,-297 40.68,-281.02 40.68,-276.98 41.5,-261 42.19,-247.52 43.65,-232.61 45.06,-220.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-220.35 46.26,-210.01 41.61,-219.53 48.56,-220.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"82\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 15665 normal</text>\n",
       "</g>\n",
       "<!-- n_d7feef994af44c2389164a5d22907b54&#45;&gt;n_10938372722d4602b4cd027a64a821d3 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>n_d7feef994af44c2389164a5d22907b54&#45;&gt;n_10938372722d4602b4cd027a64a821d3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M70.96,-347.8C87.66,-335.01 110.64,-317.42 129,-303.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"131.26,-306.03 137.07,-297.18 127,-300.48 131.26,-306.03\"/>\n",
       "<text text-anchor=\"middle\" x=\"144\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_6485735d83084dfbb08f6ad7cf4b2b37 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>n_6485735d83084dfbb08f6ad7cf4b2b37</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-471 6,-471 6,-435 91,-435 91,-471\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_6485735d83084dfbb08f6ad7cf4b2b37&#45;&gt;n_d7feef994af44c2389164a5d22907b54 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>n_6485735d83084dfbb08f6ad7cf4b2b37&#45;&gt;n_d7feef994af44c2389164a5d22907b54</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-434.8C48.5,-423.16 48.5,-407.55 48.5,-394.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-394.18 48.5,-384.18 45,-394.18 52,-394.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 15665 normal</text>\n",
       "</g>\n",
       "<!-- n_3d680bb94d9741438135050bc62003dc -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>n_3d680bb94d9741438135050bc62003dc</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"185.5,-558 131.5,-558 131.5,-522 185.5,-522 185.5,-558\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_3d680bb94d9741438135050bc62003dc&#45;&gt;n_6485735d83084dfbb08f6ad7cf4b2b37 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>n_3d680bb94d9741438135050bc62003dc&#45;&gt;n_6485735d83084dfbb08f6ad7cf4b2b37</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M136.24,-521.8C119.69,-509.01 96.92,-491.42 78.73,-477.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"80.78,-474.52 70.73,-471.18 76.5,-480.06 80.78,-474.52\"/>\n",
       "<text text-anchor=\"middle\" x=\"139.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_07bcb7c0b2834902b5d8fcd0f426b704 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>n_07bcb7c0b2834902b5d8fcd0f426b704</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-645 6,-645 6,-609 91,-609 91,-645\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_07bcb7c0b2834902b5d8fcd0f426b704&#45;&gt;n_6485735d83084dfbb08f6ad7cf4b2b37 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>n_07bcb7c0b2834902b5d8fcd0f426b704&#45;&gt;n_6485735d83084dfbb08f6ad7cf4b2b37</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-608.99C44.57,-595.26 42.39,-575.42 41.5,-558 40.68,-542.02 40.68,-537.98 41.5,-522 42.19,-508.52 43.65,-493.61 45.06,-481.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-481.35 46.26,-471.01 41.61,-480.53 48.56,-481.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"82\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 15665 normal</text>\n",
       "</g>\n",
       "<!-- n_07bcb7c0b2834902b5d8fcd0f426b704&#45;&gt;n_3d680bb94d9741438135050bc62003dc -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>n_07bcb7c0b2834902b5d8fcd0f426b704&#45;&gt;n_3d680bb94d9741438135050bc62003dc</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M70.76,-608.8C87.31,-596.01 110.08,-578.42 128.27,-564.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"130.5,-567.06 136.27,-558.18 126.22,-561.52 130.5,-567.06\"/>\n",
       "<text text-anchor=\"middle\" x=\"143\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_0ad67e0bda584079bb936c54cc7aea6b -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>n_0ad67e0bda584079bb936c54cc7aea6b</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"185.5,-732 131.5,-732 131.5,-696 185.5,-696 185.5,-732\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_0ad67e0bda584079bb936c54cc7aea6b&#45;&gt;n_07bcb7c0b2834902b5d8fcd0f426b704 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>n_0ad67e0bda584079bb936c54cc7aea6b&#45;&gt;n_07bcb7c0b2834902b5d8fcd0f426b704</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M136.24,-695.8C119.69,-683.01 96.92,-665.42 78.73,-651.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"80.78,-648.52 70.73,-645.18 76.5,-654.06 80.78,-648.52\"/>\n",
       "<text text-anchor=\"middle\" x=\"139.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_a61a81284f724b8caa90fe3cf5924c2d -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>n_a61a81284f724b8caa90fe3cf5924c2d</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-819 6,-819 6,-783 91,-783 91,-819\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_a61a81284f724b8caa90fe3cf5924c2d&#45;&gt;n_07bcb7c0b2834902b5d8fcd0f426b704 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>n_a61a81284f724b8caa90fe3cf5924c2d&#45;&gt;n_07bcb7c0b2834902b5d8fcd0f426b704</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-782.99C44.57,-769.26 42.39,-749.42 41.5,-732 40.68,-716.02 40.68,-711.98 41.5,-696 42.19,-682.52 43.65,-667.61 45.06,-655.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-655.35 46.26,-645.01 41.61,-654.53 48.56,-655.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"82\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 15665 normal</text>\n",
       "</g>\n",
       "<!-- n_a61a81284f724b8caa90fe3cf5924c2d&#45;&gt;n_0ad67e0bda584079bb936c54cc7aea6b -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>n_a61a81284f724b8caa90fe3cf5924c2d&#45;&gt;n_0ad67e0bda584079bb936c54cc7aea6b</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M70.76,-782.8C87.31,-770.01 110.08,-752.42 128.27,-738.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"130.5,-741.06 136.27,-732.18 126.22,-735.52 130.5,-741.06\"/>\n",
       "<text text-anchor=\"middle\" x=\"143\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_7b33ff551334485eb370d44bf8f8375c -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>n_7b33ff551334485eb370d44bf8f8375c</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"185.5,-906 131.5,-906 131.5,-870 185.5,-870 185.5,-906\"/>\n",
       "<text text-anchor=\"middle\" x=\"158.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_7b33ff551334485eb370d44bf8f8375c&#45;&gt;n_a61a81284f724b8caa90fe3cf5924c2d -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>n_7b33ff551334485eb370d44bf8f8375c&#45;&gt;n_a61a81284f724b8caa90fe3cf5924c2d</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M136.24,-869.8C119.69,-857.01 96.92,-839.42 78.73,-825.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"80.78,-822.52 70.73,-819.18 76.5,-828.06 80.78,-822.52\"/>\n",
       "<text text-anchor=\"middle\" x=\"139.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_3b0f0571a7a44b72a8a9d96461332e84 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>n_3b0f0571a7a44b72a8a9d96461332e84</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-993 6,-993 6,-957 91,-957 91,-993\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_3b0f0571a7a44b72a8a9d96461332e84&#45;&gt;n_a61a81284f724b8caa90fe3cf5924c2d -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>n_3b0f0571a7a44b72a8a9d96461332e84&#45;&gt;n_a61a81284f724b8caa90fe3cf5924c2d</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-956.99C44.57,-943.26 42.39,-923.42 41.5,-906 40.68,-890.02 40.68,-885.98 41.5,-870 42.19,-856.52 43.65,-841.61 45.06,-829.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-829.35 46.26,-819.01 41.61,-828.53 48.56,-829.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"82\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 15665 normal</text>\n",
       "</g>\n",
       "<!-- n_3b0f0571a7a44b72a8a9d96461332e84&#45;&gt;n_7b33ff551334485eb370d44bf8f8375c -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>n_3b0f0571a7a44b72a8a9d96461332e84&#45;&gt;n_7b33ff551334485eb370d44bf8f8375c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M70.76,-956.8C87.31,-944.01 110.08,-926.42 128.27,-912.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"130.5,-915.06 136.27,-906.18 126.22,-909.52 130.5,-915.06\"/>\n",
       "<text text-anchor=\"middle\" x=\"143\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_c1a608f088284dbea040eaef27e49dba -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>n_c1a608f088284dbea040eaef27e49dba</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-1080 21.5,-1080 21.5,-1044 75.5,-1044 75.5,-1080\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_c1a608f088284dbea040eaef27e49dba&#45;&gt;n_3b0f0571a7a44b72a8a9d96461332e84 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>n_c1a608f088284dbea040eaef27e49dba&#45;&gt;n_3b0f0571a7a44b72a8a9d96461332e84</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1043.8C48.5,-1032.16 48.5,-1016.55 48.5,-1003.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1003.18 48.5,-993.18 45,-1003.18 52,-1003.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 15665 normal</text>\n",
       "</g>\n",
       "<!-- n_22265d1270704ff6abbac058f00b1fc8 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>n_22265d1270704ff6abbac058f00b1fc8</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-1167 6,-1167 6,-1131 91,-1131 91,-1167\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_22265d1270704ff6abbac058f00b1fc8&#45;&gt;n_c1a608f088284dbea040eaef27e49dba -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>n_22265d1270704ff6abbac058f00b1fc8&#45;&gt;n_c1a608f088284dbea040eaef27e49dba</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1130.8C48.5,-1119.16 48.5,-1103.55 48.5,-1090.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1090.18 48.5,-1080.18 45,-1090.18 52,-1090.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 38116 normal</text>\n",
       "</g>\n",
       "<!-- n_b2dd7701067140edba373ab1ee710078 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>n_b2dd7701067140edba373ab1ee710078</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-1254 21.5,-1254 21.5,-1218 75.5,-1218 75.5,-1254\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1232.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_b2dd7701067140edba373ab1ee710078&#45;&gt;n_22265d1270704ff6abbac058f00b1fc8 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>n_b2dd7701067140edba373ab1ee710078&#45;&gt;n_22265d1270704ff6abbac058f00b1fc8</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1217.8C48.5,-1206.16 48.5,-1190.55 48.5,-1177.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1177.18 48.5,-1167.18 45,-1177.18 52,-1177.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1188.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 38116 normal</text>\n",
       "</g>\n",
       "<!-- n_3dc757cdd3484ef7b434b6ec9790d897 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>n_3dc757cdd3484ef7b434b6ec9790d897</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"138.5,-1341 84.5,-1341 84.5,-1305 138.5,-1305 138.5,-1341\"/>\n",
       "<text text-anchor=\"middle\" x=\"111.5\" y=\"-1319.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_3dc757cdd3484ef7b434b6ec9790d897&#45;&gt;n_b2dd7701067140edba373ab1ee710078 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>n_3dc757cdd3484ef7b434b6ec9790d897&#45;&gt;n_b2dd7701067140edba373ab1ee710078</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M98.75,-1304.8C89.7,-1292.59 77.4,-1275.99 67.24,-1262.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"70,-1260.13 61.23,-1254.18 64.37,-1264.29 70,-1260.13\"/>\n",
       "<text text-anchor=\"middle\" x=\"113.5\" y=\"-1275.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_2c2552ca1b934bac8c877134bc8b5edd -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>n_2c2552ca1b934bac8c877134bc8b5edd</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"140.5,-1428 84.5,-1428 84.5,-1392 140.5,-1392 140.5,-1428\"/>\n",
       "<text text-anchor=\"middle\" x=\"112.5\" y=\"-1406.3\" font-family=\"Times,serif\" font-size=\"14.00\">resolve</text>\n",
       "</g>\n",
       "<!-- n_2c2552ca1b934bac8c877134bc8b5edd&#45;&gt;n_3dc757cdd3484ef7b434b6ec9790d897 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>n_2c2552ca1b934bac8c877134bc8b5edd&#45;&gt;n_3dc757cdd3484ef7b434b6ec9790d897</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M113.89,-1391.87C114.49,-1382.22 114.97,-1369.94 114.5,-1359 114.39,-1356.44 114.24,-1353.78 114.05,-1351.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"117.54,-1350.76 113.26,-1341.07 110.56,-1351.32 117.54,-1350.76\"/>\n",
       "<text text-anchor=\"middle\" x=\"192\" y=\"-1362.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;9635 resolved, 0 exception</text>\n",
       "</g>\n",
       "<!-- n_23b7a4551b00456391d8920af663cc80 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>n_23b7a4551b00456391d8920af663cc80</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-1515 6,-1515 6,-1479 91,-1479 91,-1515\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1493.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_23b7a4551b00456391d8920af663cc80&#45;&gt;n_b2dd7701067140edba373ab1ee710078 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>n_23b7a4551b00456391d8920af663cc80&#45;&gt;n_b2dd7701067140edba373ab1ee710078</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.95,-1478.72C40.3,-1455.17 32.41,-1411.61 29.5,-1374 26.54,-1335.75 34.68,-1291.87 41.21,-1264.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"44.7,-1264.78 43.7,-1254.23 37.9,-1263.1 44.7,-1264.78\"/>\n",
       "<text text-anchor=\"middle\" x=\"70\" y=\"-1362.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 28935 normal</text>\n",
       "</g>\n",
       "<!-- n_23b7a4551b00456391d8920af663cc80&#45;&gt;n_2c2552ca1b934bac8c877134bc8b5edd -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>n_23b7a4551b00456391d8920af663cc80&#45;&gt;n_2c2552ca1b934bac8c877134bc8b5edd</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M61.45,-1478.8C70.65,-1466.59 83.14,-1449.99 93.47,-1436.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"96.35,-1438.27 99.57,-1428.18 90.76,-1434.06 96.35,-1438.27\"/>\n",
       "<text text-anchor=\"middle\" x=\"127.5\" y=\"-1449.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 9635 exception</text>\n",
       "</g>\n",
       "<!-- n_2b42e35faaf749adb07660f1622b1ce8 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>n_2b42e35faaf749adb07660f1622b1ce8</title>\n",
       "<ellipse fill=\"lightblue\" stroke=\"black\" cx=\"48.5\" cy=\"-1584\" rx=\"37.89\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1580.3\" font-family=\"Times,serif\" font-size=\"14.00\">readcsv</text>\n",
       "</g>\n",
       "<!-- n_2b42e35faaf749adb07660f1622b1ce8&#45;&gt;n_23b7a4551b00456391d8920af663cc80 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>n_2b42e35faaf749adb07660f1622b1ce8&#45;&gt;n_23b7a4551b00456391d8920af663cc80</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1565.8C48.5,-1554.16 48.5,-1538.55 48.5,-1525.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1525.18 48.5,-1515.18 45,-1525.18 52,-1525.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1536.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 38570 normal</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<flowgiston.base.FlowgistonChart at 0x12c73cf70>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_chart(steps[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.2 clean data (no exceptions, no ignores, no resolves should happen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> reading input files\n",
      ">>> remove rows causing exceptions\n",
      ">>> replicating data 1x\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00, 27.46it/s]\n",
      "done.\n"
     ]
    }
   ],
   "source": [
    "!cd .. && python3 preprocess-data.py -s 1 -m clean -o data/zillow_clean.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data/zillow_clean.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps = zillow_pipeline(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.44.1 (20200629.0846)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"258pt\" height=\"1610pt\"\n",
       " viewBox=\"0.00 0.00 257.50 1610.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 1606)\">\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-1606 253.5,-1606 253.5,4 -4,4\"/>\n",
       "<!-- n_5ff251ea85d24ea592e17cd602b76535 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>n_5ff251ea85d24ea592e17cd602b76535</title>\n",
       "<ellipse fill=\"lightblue\" stroke=\"black\" cx=\"48.5\" cy=\"-18\" rx=\"29.5\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">tocsv</text>\n",
       "</g>\n",
       "<!-- n_a2f1fd8d983741edadbcc637574fafee -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>n_a2f1fd8d983741edadbcc637574fafee</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"97,-123 0,-123 0,-87 97,-87 97,-123\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\">selectColumns</text>\n",
       "</g>\n",
       "<!-- n_a2f1fd8d983741edadbcc637574fafee&#45;&gt;n_5ff251ea85d24ea592e17cd602b76535 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>n_a2f1fd8d983741edadbcc637574fafee&#45;&gt;n_5ff251ea85d24ea592e17cd602b76535</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-86.8C48.5,-75.16 48.5,-59.55 48.5,-46.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-46.18 48.5,-36.18 45,-46.18 52,-46.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1211 normal</text>\n",
       "</g>\n",
       "<!-- n_fef456f032544410a77843e9d3158efe -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>n_fef456f032544410a77843e9d3158efe</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-210 21.5,-210 21.5,-174 75.5,-174 75.5,-210\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_fef456f032544410a77843e9d3158efe&#45;&gt;n_a2f1fd8d983741edadbcc637574fafee -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>n_fef456f032544410a77843e9d3158efe&#45;&gt;n_a2f1fd8d983741edadbcc637574fafee</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-173.8C48.5,-162.16 48.5,-146.55 48.5,-133.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-133.18 48.5,-123.18 45,-133.18 52,-133.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 1211 normal</text>\n",
       "</g>\n",
       "<!-- n_b06bd2d700a543cba3fb456d558015a1 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>n_b06bd2d700a543cba3fb456d558015a1</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"180.5,-297 124.5,-297 124.5,-261 180.5,-261 180.5,-297\"/>\n",
       "<text text-anchor=\"middle\" x=\"152.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\">resolve</text>\n",
       "</g>\n",
       "<!-- n_b06bd2d700a543cba3fb456d558015a1&#45;&gt;n_fef456f032544410a77843e9d3158efe -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>n_b06bd2d700a543cba3fb456d558015a1&#45;&gt;n_fef456f032544410a77843e9d3158efe</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M131.45,-260.8C115.95,-248.13 94.67,-230.73 77.54,-216.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.47,-213.79 69.51,-210.18 75.04,-219.21 79.47,-213.79\"/>\n",
       "<text text-anchor=\"middle\" x=\"174\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 resolved, 0 exception</text>\n",
       "</g>\n",
       "<!-- n_baebc3ad131d4574906704411ac2e3e9 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>n_baebc3ad131d4574906704411ac2e3e9</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-384 6,-384 6,-348 91,-348 91,-384\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_baebc3ad131d4574906704411ac2e3e9&#45;&gt;n_fef456f032544410a77843e9d3158efe -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>n_baebc3ad131d4574906704411ac2e3e9&#45;&gt;n_fef456f032544410a77843e9d3158efe</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-347.99C44.57,-334.26 42.39,-314.42 41.5,-297 40.68,-281.02 40.68,-276.98 41.5,-261 42.19,-247.52 43.65,-232.61 45.06,-220.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-220.35 46.26,-210.01 41.61,-219.53 48.56,-220.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 6030 normal</text>\n",
       "</g>\n",
       "<!-- n_baebc3ad131d4574906704411ac2e3e9&#45;&gt;n_b06bd2d700a543cba3fb456d558015a1 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>n_baebc3ad131d4574906704411ac2e3e9&#45;&gt;n_b06bd2d700a543cba3fb456d558015a1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.55,-347.8C85.05,-335.13 106.33,-317.73 123.46,-303.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.96,-306.21 131.49,-297.18 121.53,-300.79 125.96,-306.21\"/>\n",
       "<text text-anchor=\"middle\" x=\"140\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_c74349a775424282ba36c45298a1fb17 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>n_c74349a775424282ba36c45298a1fb17</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-471 6,-471 6,-435 91,-435 91,-471\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_c74349a775424282ba36c45298a1fb17&#45;&gt;n_baebc3ad131d4574906704411ac2e3e9 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>n_c74349a775424282ba36c45298a1fb17&#45;&gt;n_baebc3ad131d4574906704411ac2e3e9</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-434.8C48.5,-423.16 48.5,-407.55 48.5,-394.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-394.18 48.5,-384.18 45,-394.18 52,-394.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 6030 normal</text>\n",
       "</g>\n",
       "<!-- n_04c70b68055f48a188dead37ad4b00ab -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>n_04c70b68055f48a188dead37ad4b00ab</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"178.5,-558 124.5,-558 124.5,-522 178.5,-522 178.5,-558\"/>\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_04c70b68055f48a188dead37ad4b00ab&#45;&gt;n_c74349a775424282ba36c45298a1fb17 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>n_04c70b68055f48a188dead37ad4b00ab&#45;&gt;n_c74349a775424282ba36c45298a1fb17</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M130.66,-521.8C115.3,-509.13 94.22,-491.73 77.26,-477.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.25,-474.84 69.31,-471.18 74.8,-480.24 79.25,-474.84\"/>\n",
       "<text text-anchor=\"middle\" x=\"135.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_ce938406a0504cbc89e19e58a58887cd -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>n_ce938406a0504cbc89e19e58a58887cd</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-645 6,-645 6,-609 91,-609 91,-645\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_ce938406a0504cbc89e19e58a58887cd&#45;&gt;n_c74349a775424282ba36c45298a1fb17 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>n_ce938406a0504cbc89e19e58a58887cd&#45;&gt;n_c74349a775424282ba36c45298a1fb17</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-608.99C44.57,-595.26 42.39,-575.42 41.5,-558 40.68,-542.02 40.68,-537.98 41.5,-522 42.19,-508.52 43.65,-493.61 45.06,-481.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-481.35 46.26,-471.01 41.61,-480.53 48.56,-481.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 6030 normal</text>\n",
       "</g>\n",
       "<!-- n_ce938406a0504cbc89e19e58a58887cd&#45;&gt;n_04c70b68055f48a188dead37ad4b00ab -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>n_ce938406a0504cbc89e19e58a58887cd&#45;&gt;n_04c70b68055f48a188dead37ad4b00ab</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.34,-608.8C84.7,-596.13 105.78,-578.73 122.74,-564.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.2,-567.24 130.69,-558.18 120.75,-561.84 125.2,-567.24\"/>\n",
       "<text text-anchor=\"middle\" x=\"139\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_dec1893caabe4963b519b8ad7cdae270 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>n_dec1893caabe4963b519b8ad7cdae270</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"178.5,-732 124.5,-732 124.5,-696 178.5,-696 178.5,-732\"/>\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_dec1893caabe4963b519b8ad7cdae270&#45;&gt;n_ce938406a0504cbc89e19e58a58887cd -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>n_dec1893caabe4963b519b8ad7cdae270&#45;&gt;n_ce938406a0504cbc89e19e58a58887cd</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M130.66,-695.8C115.3,-683.13 94.22,-665.73 77.26,-651.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.25,-648.84 69.31,-645.18 74.8,-654.24 79.25,-648.84\"/>\n",
       "<text text-anchor=\"middle\" x=\"135.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_cd945c78237b4241b476a69b30d44e98 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>n_cd945c78237b4241b476a69b30d44e98</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-819 6,-819 6,-783 91,-783 91,-819\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-797.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_cd945c78237b4241b476a69b30d44e98&#45;&gt;n_ce938406a0504cbc89e19e58a58887cd -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>n_cd945c78237b4241b476a69b30d44e98&#45;&gt;n_ce938406a0504cbc89e19e58a58887cd</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-782.99C44.57,-769.26 42.39,-749.42 41.5,-732 40.68,-716.02 40.68,-711.98 41.5,-696 42.19,-682.52 43.65,-667.61 45.06,-655.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-655.35 46.26,-645.01 41.61,-654.53 48.56,-655.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 6030 normal</text>\n",
       "</g>\n",
       "<!-- n_cd945c78237b4241b476a69b30d44e98&#45;&gt;n_dec1893caabe4963b519b8ad7cdae270 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>n_cd945c78237b4241b476a69b30d44e98&#45;&gt;n_dec1893caabe4963b519b8ad7cdae270</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.34,-782.8C84.7,-770.13 105.78,-752.73 122.74,-738.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.2,-741.24 130.69,-732.18 120.75,-735.84 125.2,-741.24\"/>\n",
       "<text text-anchor=\"middle\" x=\"139\" y=\"-753.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_9d78579e20a544de841c7398cc19255c -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>n_9d78579e20a544de841c7398cc19255c</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"178.5,-906 124.5,-906 124.5,-870 178.5,-870 178.5,-906\"/>\n",
       "<text text-anchor=\"middle\" x=\"151.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_9d78579e20a544de841c7398cc19255c&#45;&gt;n_cd945c78237b4241b476a69b30d44e98 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>n_9d78579e20a544de841c7398cc19255c&#45;&gt;n_cd945c78237b4241b476a69b30d44e98</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M130.66,-869.8C115.3,-857.13 94.22,-839.73 77.26,-825.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.25,-822.84 69.31,-819.18 74.8,-828.24 79.25,-822.84\"/>\n",
       "<text text-anchor=\"middle\" x=\"135.5\" y=\"-840.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_330a48980b454c1a9de9e074bcc46a9b -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>n_330a48980b454c1a9de9e074bcc46a9b</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-993 6,-993 6,-957 91,-957 91,-993\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-971.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_330a48980b454c1a9de9e074bcc46a9b&#45;&gt;n_cd945c78237b4241b476a69b30d44e98 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>n_330a48980b454c1a9de9e074bcc46a9b&#45;&gt;n_cd945c78237b4241b476a69b30d44e98</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M46.26,-956.99C44.57,-943.26 42.39,-923.42 41.5,-906 40.68,-890.02 40.68,-885.98 41.5,-870 42.19,-856.52 43.65,-841.61 45.06,-829.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"48.56,-829.35 46.26,-819.01 41.61,-828.53 48.56,-829.35\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-884.3\" font-family=\"Times,serif\" font-size=\"14.00\"> 6030 normal</text>\n",
       "</g>\n",
       "<!-- n_330a48980b454c1a9de9e074bcc46a9b&#45;&gt;n_9d78579e20a544de841c7398cc19255c -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>n_330a48980b454c1a9de9e074bcc46a9b&#45;&gt;n_9d78579e20a544de841c7398cc19255c</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M69.34,-956.8C84.7,-944.13 105.78,-926.73 122.74,-912.74\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"125.2,-915.24 130.69,-906.18 120.75,-909.84 125.2,-915.24\"/>\n",
       "<text text-anchor=\"middle\" x=\"139\" y=\"-927.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_2e3c9c1ec2a94907a8de7365c9638781 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>n_2e3c9c1ec2a94907a8de7365c9638781</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-1080 21.5,-1080 21.5,-1044 75.5,-1044 75.5,-1080\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1058.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_2e3c9c1ec2a94907a8de7365c9638781&#45;&gt;n_330a48980b454c1a9de9e074bcc46a9b -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>n_2e3c9c1ec2a94907a8de7365c9638781&#45;&gt;n_330a48980b454c1a9de9e074bcc46a9b</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1043.8C48.5,-1032.16 48.5,-1016.55 48.5,-1003.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1003.18 48.5,-993.18 45,-1003.18 52,-1003.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"85.5\" y=\"-1014.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 6030 normal</text>\n",
       "</g>\n",
       "<!-- n_bdba59d8793a4a8189e725f69958667d -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>n_bdba59d8793a4a8189e725f69958667d</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-1167 6,-1167 6,-1131 91,-1131 91,-1167\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1145.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_bdba59d8793a4a8189e725f69958667d&#45;&gt;n_2e3c9c1ec2a94907a8de7365c9638781 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>n_bdba59d8793a4a8189e725f69958667d&#45;&gt;n_2e3c9c1ec2a94907a8de7365c9638781</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1130.8C48.5,-1119.16 48.5,-1103.55 48.5,-1090.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1090.18 48.5,-1080.18 45,-1090.18 52,-1090.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1101.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 28481 normal</text>\n",
       "</g>\n",
       "<!-- n_c23589743c944ba0bb6eb4031d0ac966 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>n_c23589743c944ba0bb6eb4031d0ac966</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"75.5,-1254 21.5,-1254 21.5,-1218 75.5,-1218 75.5,-1254\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1232.3\" font-family=\"Times,serif\" font-size=\"14.00\">filter</text>\n",
       "</g>\n",
       "<!-- n_c23589743c944ba0bb6eb4031d0ac966&#45;&gt;n_bdba59d8793a4a8189e725f69958667d -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>n_c23589743c944ba0bb6eb4031d0ac966&#45;&gt;n_bdba59d8793a4a8189e725f69958667d</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1217.8C48.5,-1206.16 48.5,-1190.55 48.5,-1177.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1177.18 48.5,-1167.18 45,-1177.18 52,-1177.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1188.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 28481 normal</text>\n",
       "</g>\n",
       "<!-- n_79802db2e6ae4cdbbfbd2fcc8c1628fe -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>n_79802db2e6ae4cdbbfbd2fcc8c1628fe</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"138.5,-1341 84.5,-1341 84.5,-1305 138.5,-1305 138.5,-1341\"/>\n",
       "<text text-anchor=\"middle\" x=\"111.5\" y=\"-1319.3\" font-family=\"Times,serif\" font-size=\"14.00\">ignore</text>\n",
       "</g>\n",
       "<!-- n_79802db2e6ae4cdbbfbd2fcc8c1628fe&#45;&gt;n_c23589743c944ba0bb6eb4031d0ac966 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>n_79802db2e6ae4cdbbfbd2fcc8c1628fe&#45;&gt;n_c23589743c944ba0bb6eb4031d0ac966</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M98.75,-1304.8C89.7,-1292.59 77.4,-1275.99 67.24,-1262.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"70,-1260.13 61.23,-1254.18 64.37,-1264.29 70,-1260.13\"/>\n",
       "<text text-anchor=\"middle\" x=\"113.5\" y=\"-1275.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 ignored</text>\n",
       "</g>\n",
       "<!-- n_332c7bb5961145618b7c7901ad9fd4cc -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>n_332c7bb5961145618b7c7901ad9fd4cc</title>\n",
       "<polygon fill=\"lightyellow\" stroke=\"black\" points=\"140.5,-1428 84.5,-1428 84.5,-1392 140.5,-1392 140.5,-1428\"/>\n",
       "<text text-anchor=\"middle\" x=\"112.5\" y=\"-1406.3\" font-family=\"Times,serif\" font-size=\"14.00\">resolve</text>\n",
       "</g>\n",
       "<!-- n_332c7bb5961145618b7c7901ad9fd4cc&#45;&gt;n_79802db2e6ae4cdbbfbd2fcc8c1628fe -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>n_332c7bb5961145618b7c7901ad9fd4cc&#45;&gt;n_79802db2e6ae4cdbbfbd2fcc8c1628fe</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M113.89,-1391.87C114.49,-1382.22 114.97,-1369.94 114.5,-1359 114.39,-1356.44 114.24,-1353.78 114.05,-1351.12\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"117.54,-1350.76 113.26,-1341.07 110.56,-1351.32 117.54,-1350.76\"/>\n",
       "<text text-anchor=\"middle\" x=\"182\" y=\"-1362.8\" font-family=\"Times,serif\" font-size=\"14.00\"> &#160;0 resolved, 0 exception</text>\n",
       "</g>\n",
       "<!-- n_3c92bfb753cb4ba880068112fef5a1cf -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>n_3c92bfb753cb4ba880068112fef5a1cf</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"91,-1515 6,-1515 6,-1479 91,-1479 91,-1515\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1493.3\" font-family=\"Times,serif\" font-size=\"14.00\">withColumn</text>\n",
       "</g>\n",
       "<!-- n_3c92bfb753cb4ba880068112fef5a1cf&#45;&gt;n_c23589743c944ba0bb6eb4031d0ac966 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>n_3c92bfb753cb4ba880068112fef5a1cf&#45;&gt;n_c23589743c944ba0bb6eb4031d0ac966</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.95,-1478.72C40.3,-1455.17 32.41,-1411.61 29.5,-1374 26.54,-1335.75 34.68,-1291.87 41.21,-1264.3\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"44.7,-1264.78 43.7,-1254.23 37.9,-1263.1 44.7,-1264.78\"/>\n",
       "<text text-anchor=\"middle\" x=\"70\" y=\"-1362.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 28935 normal</text>\n",
       "</g>\n",
       "<!-- n_3c92bfb753cb4ba880068112fef5a1cf&#45;&gt;n_332c7bb5961145618b7c7901ad9fd4cc -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>n_3c92bfb753cb4ba880068112fef5a1cf&#45;&gt;n_332c7bb5961145618b7c7901ad9fd4cc</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M61.45,-1478.8C70.65,-1466.59 83.14,-1449.99 93.47,-1436.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"96.35,-1438.27 99.57,-1428.18 90.76,-1434.06 96.35,-1438.27\"/>\n",
       "<text text-anchor=\"middle\" x=\"117\" y=\"-1449.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 0 exception</text>\n",
       "</g>\n",
       "<!-- n_2efc245dd3ef4e5bb491531708b645c8 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>n_2efc245dd3ef4e5bb491531708b645c8</title>\n",
       "<ellipse fill=\"lightblue\" stroke=\"black\" cx=\"48.5\" cy=\"-1584\" rx=\"37.89\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-1580.3\" font-family=\"Times,serif\" font-size=\"14.00\">readcsv</text>\n",
       "</g>\n",
       "<!-- n_2efc245dd3ef4e5bb491531708b645c8&#45;&gt;n_3c92bfb753cb4ba880068112fef5a1cf -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>n_2efc245dd3ef4e5bb491531708b645c8&#45;&gt;n_3c92bfb753cb4ba880068112fef5a1cf</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.5,-1565.8C48.5,-1554.16 48.5,-1538.55 48.5,-1525.24\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"52,-1525.18 48.5,-1515.18 45,-1525.18 52,-1525.18\"/>\n",
       "<text text-anchor=\"middle\" x=\"89\" y=\"-1536.8\" font-family=\"Times,serif\" font-size=\"14.00\"> 28935 normal</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<flowgiston.base.FlowgistonChart at 0x130c4a910>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_chart(steps[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
